import datetime
import json
import shutil
import sys
from functools import lru_cache

import demjson
from pymysql.cursors import Cursor

from TuShare.TUShareObject import TUShareObject

sys.path.extend(["F:/ML/BNC", "D:/BNC", "/www/wwwroot/BNC"])

from Utils.Functions import dateRange1
from DataBase import DatabaseConsts
from DataBase.DatabaseConsts import CodeTagDateWeightTempTableName, CodeTagDateTableName, StockBasicTableName, \
    StockTagRenameTableName, StockTagManagerDB
from DataBase.DBPoolObject import DB_POOL

MAX_NUM = sys.maxsize - 100
MIN_NUM = -(MAX_NUM - 10)

# DB_pool = DB_POOL.get_db_pool(host=MySQLServerIP_199,
#                               port=3306,
#                               user='root',
#                               passwd='root',
#                               db=StockCalendarDB,
#                               charset='utf8',
#                               cursor_class=Cursor)

DB_pool = DB_POOL.get_db_pool(db=StockTagManagerDB,
                              host='localhost',
                              passwd=DatabaseConsts.MYSQLServerPW_H_Local,
                              cursor_class=Cursor)

connection = DB_pool.connection()
cursor = connection.cursor()
JSON_FILE_PATH = "F:/ML/BNC/WebServer/static/StockCalendar.json"
BASE_DATE_INT_FORMAT = "%Y%m%d"
# alter 这个数据不能低于start_date_default
trade_start_date = "20151001"
DAILY_MIN_CODE_NUM = 5
trade_end_date = datetime.date.today().strftime(BASE_DATE_INT_FORMAT)

MIN_DATE_INT = 00000000
MAX_DATE_INT = 99999999
DATA_START_DATE = 20170101
start_date_default, end_date_default = [DATA_START_DATE, MAX_DATE_INT]
skip_tag_set = {"新股", "低价", "次新", "超跌", "次新股", "股权转让", "高送转", "并购", "填权", "公告", "庄股", "18年报股", "上市新股", "业绩预增", "复牌",
                "业绩", "妖股", "新股上市", "老妖", "股权", "涨价", "高送转预期", "定增", "回购", "超跌反弹", "业绩增长", "举牌", "17年报高送转填权", "首批",
                "价值重估"}
not_allowed_contain_word = {"季报", "低价", "年报", "月报"}


# code 使用的格式为 000000 date 使用格式为 "%Y%m%d"
def select_cur_date_from_db():
    # 选取当前最大日期
    query_di = {"ctwd_tb": CodeTagDateWeightTempTableName, "st_db": StockTagManagerDB}
    sql = "select max(cur_date) from {st_db}.{ctwd_tb}".format(**query_di)
    cursor.execute(sql)
    r = cursor.fetchone()
    if r and r[0]:
        return r[0]
    return MIN_DATE_INT


def select_cur_date_from_json():
    # 读取json文件
    dic = select_calendar_from_json()
    if dic:
        date_i = -1
        for one_day_date in dic:
            date_i = max(date_i, int(one_day_date['date']))
        date_s = str(date_i)
        return int(date_s)
    return MIN_DATE_INT


def select_one_day_code_tag_status_from_db(cur_date__):
    """ 选取当日的最少的信息 """
    # sql = "SELECT DISTINCT CODE, IFNULL(TR.TO,CTD.tag) AS tag,date FROM {} AS CTD LEFT OUTER JOIN {} AS TR ON TR.dispose= {} AND CTD.tag = TR.from WHERE date = %s ORDER BY CODE".format(
    #     CodeTagDateTableName, StockTagRenameTableName, NOT_DISPOSE)

    sql = "SELECT DISTINCT CODE, IFNULL(TR.TO,CTD.tag) AS tag,date FROM {} AS CTD LEFT OUTER JOIN {} AS TR ON CTD.tag = TR.origin WHERE date = %s ORDER BY CODE".format(
        CodeTagDateTableName, StockTagRenameTableName)
    cursor.execute(sql, [cur_date__])
    return cursor.fetchall()


tag_eailies_appear_time = {}


def init_tag_ea_calendar():
    """ 使用code_tag_weight 初始化标签日历 """
    # select_one_table_tag_format_ = "select MIN(date),IFNULL(TR.to, CTD.tag) as tag1 from {} AS CTD LEFT OUTER JOIN {} AS TR ON TR.dispose = {} AND TR.`from` = CTD.tag group by tag1".format(
    #     CodeTagDateTableName, StockTagRenameTableName, NOT_DISPOSE)

    select_one_table_tag_format_ = "select MIN(date),IFNULL(TR.to, CTD.tag) as tag1 from {} AS CTD LEFT OUTER JOIN {} AS TR ON TR.`origin` = CTD.tag group by tag1".format(
        CodeTagDateTableName, StockTagRenameTableName)
    cursor.execute(select_one_table_tag_format_)
    res = cursor.fetchall()
    for [dte_, tag_] in res:
        tag_eailies_appear_time[tag_] = int(dte_.strftime(BASE_DATE_INT_FORMAT))
    print("初始化标签出现的最早时间完成")


@lru_cache(maxsize=6000)
def get_code_start_time(code) -> int:
    """ 需要输出标准的date int格式 "%Y%m%d" """

    select_code_end_time = "select list_date from {} where symbol = %s"
    cursor.execute(select_code_end_time.format(DatabaseConsts.StockBasicTableName), [code])
    res = cursor.fetchone()

    if res and res[0]:
        date_s = res[0]
        res_date = int(date_s)
    else:
        res_date = MIN_DATE_INT
    return res_date


@lru_cache(maxsize=6000)
def get_code_end_time(code) -> int:
    """ 返回标准的股票的结束时间

        return
            int : BASE_DATE_INT_FORMAT 格式的整数
    """
    select_code_end_time = "select delist_date from {} where ts_code = %s"
    cursor.execute(select_code_end_time.format(DatabaseConsts.StockBasicTableName), [code])
    res = cursor.fetchone()
    if res and res[0]:
        date_s = res[0]
        res_date = int(date_s)
    else:
        res_date = MAX_DATE_INT
    return res_date


tag_etd = MAX_NUM


@lru_cache(maxsize=10000)
def select_code_tag_life(code_, tag_, min_start_date, max_end_date):
    """ 获取code_tag 的周期 """

    code_tag_start_time = get_code_start_time(code_)
    code_tag_end_time = get_code_end_time(code_)

    if tag_ not in tag_eailies_appear_time:
        tag_std = MIN_NUM
    else:
        tag_std = tag_eailies_appear_time[tag_]
    rstd = max(tag_std, code_tag_start_time, min_start_date)
    retd = min(tag_etd, code_tag_end_time, max_end_date)
    return [rstd, retd]


def select_previous_CTDW_from_db(max_date):
    """ 这里的标签已经标准化， 不需要进行转化 """
    query_di = {"ctwd_tb": CodeTagDateWeightTempTableName, "st_db": StockTagManagerDB}
    sql = "select code, tag, cur_up_times, cur_date,cur_count from {st_db}.{ctwd_tb} where cal_date = %s".format(
        **query_di)
    cursor.execute(sql, [max_date])
    return cursor.fetchall()


@lru_cache()
def select_calendar_from_json():
    """ 从json文件中读取对应的数据信息， 只是用于读取数据"""
    dic = dict()
    with open(JSON_FILE_PATH, mode='r', encoding='utf8') as f:
        jso_s = f.read()
    if jso_s:
        dic = demjson.decode(jso_s)
    if "data" in dic:
        return dic['data']
    return list()


def find_this_tag_at_today(temp_calendar_dic, tag_):
    """ 查找list中属性的对应"""
    for one_tag_inc in temp_calendar_dic['tags_inc']:
        if one_tag_inc['tag'] == tag_:
            return [True, one_tag_inc]
    return False, None


def select_code_name(code_: str) -> str:
    """ 查找code_ 对应的tag

        :argument
            code_ : 格式为 \d{6}
        :returns
            name_ : 股票的名字，需要带上ST等特殊标志
    """
    # fixme ST的股票是动态变化的
    # 需要注意格式
    # tushare中的数据可能是不保存退市的股票，需要自己进行判断
    sql = " select SB.name AS name_1 from {} AS SB where SB.symbol = %s ".format(StockBasicTableName)
    cursor.execute(sql, [code_])
    r = cursor.fetchone()
    if r and r[0]:
        return r[0]
    return "默认股票名"


pre_val = 0


def save_data_2_code_tag_weight(code_tag_statistic_dic):
    """ 保存code_tag_statistic到数据库中去 """
    values_db = list()
    # 保存到DB中，需要清除之前的数据
    # truncate_sql = "truncate {} ;".format(CodeTagDateWeightTempTableName)
    insert_sql = "REPLACE INTO {} (code, tag,cur_up_times, cur_date,cur_count,cal_date) VALUES (%s, %s, %s, %s, %s,%s)".format(
        CodeTagDateWeightTempTableName)
    for code_ in code_tag_statistic_dic:
        for tag_ in code_tag_statistic_dic[code_]:
            code_tag_rela = code_tag_statistic_dic[code_][tag_]
            new_line_data_ = (
                code_, tag_, code_tag_rela['cur_up_times'], code_tag_rela['cur_date'], code_tag_rela['cur_count'],
                code_tag_rela['cal_date'])
            values_db.append(new_line_data_)
            # "cur_up_times": cur_up_times_, "cur_date": cur_date_,
            # "cur_count": cur_date_count_
    # cursor.execute(truncate_sql)
    # connection.commit()

    print("values _db :{}".format(len(values_db)))
    # global pre_val
    # assert len(values_db) >= pre_val
    # pre_val = len(values_db)
    cursor.executemany(insert_sql, values_db)
    connection.commit()


def save_data_2_file(code_tag_calendar_list_json):
    """ 需要删除现有的json中的所有 数据 """
    # 套一个娃
    dat = dict()
    dat['data'] = code_tag_calendar_list_json
    dat["gaobiao_scale"] = 1
    dat_s = json.dumps(dat)
    with open(JSON_FILE_PATH, mode="w", encoding="UTF8") as file_:
        file_.write(dat_s)


@lru_cache(maxsize=1)
def get_tushare_trade_day_set(tsd=trade_start_date, ted=trade_end_date) -> set:
    """ 获取tushare的交易日信息， 需要包含SSE SZSE"""

    trade_day_set = set()
    pro_api = TUShareObject.instance()
    df = pro_api.trade_cal(exchange='SSE', start_date=tsd, end_date=ted, is_open=1)
    trade_day_set.update(df['cal_date'].tolist())
    df = pro_api.trade_cal(exchange='SZSE', start_date=tsd, end_date=ted, is_open=1)
    trade_day_set.update(df['cal_date'].tolist())

    return trade_day_set


@lru_cache(maxsize=10000)
def use_trade_day_combine_cur_count(code_, tag_, code_tag_start_, code_tag_end_):
    # 初始化获取tushare的数据
    trade_day_set = get_tushare_trade_day_set()
    code_tag_range_set = set(dateRange1(code_tag_start_, code_tag_end_, step=1))
    return len(trade_day_set & code_tag_range_set)


def should_skip_tag(tag_: str):
    # 判断是否含有相似的东西
    contain_smilar_word = False
    for nta in not_allowed_contain_word:
        if tag_.find(nta) != -1:
            contain_smilar_word = True
            break
    if tag_ in skip_tag_set or contain_smilar_word:
        return True
    return False


def last_date_from_CTD():
    sql = "select max(date+1) from {}".format(CodeTagDateTableName)
    cursor.execute(sql)
    r = cursor.fetchone()
    assert r, "empty table"
    print(r)
    return int(r[0])


def select_tag_rela_codes():
    """ 选取当前标签和股票的关系信息"""
    # select_sql = "select distinct code, IFNULL(TR.`to`,CTD.tag) as tag from {} AS CTD LEFT OUTER JOIN {} AS TR ON TR.dispose = {} AND TR.from = CTD.tag ".format(
    #     CodeTagDateTableName, StockTagRenameTableName, NOT_DISPOSE)

    select_sql = "select distinct code, IFNULL(TR.`to`,CTD.tag) as tag from {} AS CTD LEFT OUTER JOIN {} AS TR ON TR.origin = CTD.tag ".format(
        CodeTagDateTableName, StockTagRenameTableName)
    cursor.execute(select_sql)
    res_di = dict()
    res = cursor.fetchall()
    assert res, "tag _rela_codes 为空"
    for [code_, tag_] in res:
        res_di.setdefault(tag_, set())
        res_di[tag_].add(code_)
    return res_di


def delete_old_machine_code_tag_relaion():
    """
        删除原有的 relation关系，不能够对人的数据进行更新
     """
    pass


def refresh_replay_data_2_code_tag_relation():
    """ 刷新复盘数据到code_tag_relation中，不能够对人的数据进行更新 """
    pass


def rewrite_tag_rename():
    """ 添加之前不存在的映射信息到复盘数据中 """
    pass


def main(default_start_date=None):
    """
        全量更新：
                1、删除来自于机器的股票标签关系，不删除人为的股票-标签 关系， 为什么要删除非认为的数据，因为需要保证机器识别的数据能够正确
                2、重新刷入新的股票标签关系，不能够让机器变更人操作的数据
        增量更新：
                1、从复盘数据库中检测最新的code_tag_date刷新 股票-标签数据

     """
    try:
        # 重新计算， 不能改变人 插入|更新（复盘数据是不会变化的）？的数据 。

        code_tag_statistic_dic = dict()  # {code:{tag:{cur_up_times:"","cur_date":"","cur_count":""}}}
        # end_date_default 通过数据库表中的最新时间来确定
        start_date = ''
        # start_date = 20170410
        max_end_date = int(datetime.date.today().strftime("%Y%m%d"))
        end_date = max_end_date
        min_start_date = start_date_default

        cur_date_from_db = select_cur_date_from_db()
        # cur_date_from_json = select_cur_date_from_json()
        init_tag_ea_calendar()
        tags_rela_codes = select_tag_rela_codes()  # {t1 : {c1,c2,c3},t2:{c2,c5}}

        # code_tag_calendar_list_json = select_calendar_from_json()

        if not cur_date_from_db == MIN_DATE_INT:
            # 直接获取数据库中的内容
            # 从数据库中读取以前的数据
            old_code_tag_status_from_db = select_previous_CTDW_from_db(cur_date_from_db)
            # 只用进行赋值
            for [code_, tag_, cur_up_times_, cur_date_, cur_date_count_] in old_code_tag_status_from_db:
                # 重组为map的形式
                code_tag_statistic_dic.setdefault(code_, dict())
                code_tag_statistic_dic[code_].setdefault(tag_, {"cur_up_times": cur_up_times_, "cur_date": cur_date_,
                                                                "cur_count": cur_date_count_})
            start_date = cur_date_from_db
        else:
            start_date = min_start_date

        if default_start_date:
            start_date = default_start_date
        # start_date = 20190109
        # 获取时间的范围 需要保持时序关系， 以免后面出现问题
        trade_day_set = get_tushare_trade_day_set()
        al = set(dateRange1(start_date, end_date, cont_l=True, cont_r=True))  # 选取时已经进行 + 1
        trade_day_set_combine_own_range = list(sorted(al & trade_day_set))
        for cur_parse_date_ in trade_day_set_combine_own_range:
            print("parse date : {}".format(cur_parse_date_))

            temp_calendar_dic = dict()

            temp_tag_num_dic = dict()
            # 选出当日前几的数据信息
            code_tag_status_from_db = select_one_day_code_tag_status_from_db(cur_parse_date_)
            cur_parse_date_i = int(cur_parse_date_)
            cur_date_ = ''
            if code_tag_status_from_db:
                for [code_, tag_, cur_date_, ] in code_tag_status_from_db:
                    # 需要保存tag_的数量
                    temp_tag_num_dic.setdefault(tag_, 0)
                    temp_tag_num_dic[tag_] += 1

                # 先对全部的tag下面的code进行cur_count +=1
                for tag_ in tags_rela_codes:
                    if tag_ in temp_tag_num_dic and temp_tag_num_dic[tag_] >= DAILY_MIN_CODE_NUM:  # 当日的标签表现突出
                        for code_ in tags_rela_codes[tag_]:
                            [code_tag_start_, code_tag_end_] = select_code_tag_life(code_, tag_, min_start_date,
                                                                                    int(cur_parse_date_))
                            if code_tag_end_ >= int(cur_parse_date_) >= code_tag_start_:
                                code_tag_statistic_dic.setdefault(code_, dict())
                                code_tag_statistic_dic[code_].setdefault(tag_, {"cur_count": 0, "cur_up_times": 0,
                                                                                "cur_date": cur_parse_date_})  # 用于进行排序
                                code_tag_statistic_dic[code_][tag_]['cur_count'] += 1

                for [code_, tag_, cur_date_, ] in code_tag_status_from_db:
                    # 如果数量小于几代表涨幅不足， 对于所有的tag_下面的所有code的cur_count都应该+1,如果当日涨停的股票中有这个股票其cur_up_times也需要加一
                    # 下面不能在进行cur_count的修改
                    if not (tag_ in temp_tag_num_dic and temp_tag_num_dic[tag_] >= DAILY_MIN_CODE_NUM):
                        continue

                    cur_date_ = int(cur_date_.strftime(BASE_DATE_INT_FORMAT))
                    name_ = select_code_name(code_)
                    if name_.find("ST") >= 0 or should_skip_tag(tag_):
                        continue

                    # 重设最大的结束时间
                    # 如果当日标签爆发了才进行cur_count ++ 的操作
                    [code_tag_start_, code_tag_end_] = select_code_tag_life(code_, tag_, min_start_date, cur_date_)
                    # cur_count_of_trade = use_trade_day_combine_cur_count(code_, tag_, code_tag_start_, code_tag_end_, )
                    code_tag_statistic_dic.setdefault(code_, dict())
                    code_tag_statistic_dic[code_].setdefault(tag_, {"cur_count": 0, "cur_up_times": 0,
                                                                    "cur_date": cur_parse_date_,
                                                                    })  # 用于进行排序

                    # 进行每次的计算的更新
                    # code_tag_statistic_dic[code_][tag_]["cur_count"] = cur_count_of_trade 不再改变cur_count的值
                    c_t_status_ = code_tag_statistic_dic[code_][tag_]
                    c_t_status_["cur_date"] = cur_date_
                    # 默认设置为0
                    # c_t_status_["cur_count"] += 1

                    if code_tag_end_ >= cur_date_ >= code_tag_start_:
                        c_t_status_["cur_up_times"] += 1
                        c_t_status_["cal_date"] = cur_date_

                    # tag 的 权重使用当前多个
                    temp_calendar_dic.setdefault("date", cur_date_)
                    temp_calendar_dic.setdefault("tags_inc", list())
                    tags_inc_list = temp_calendar_dic['tags_inc']
                    # 找到现有的tag， 如果不存在就新建一个
                    has_found_tag, tag_incr_ = find_this_tag_at_today(temp_calendar_dic, tag_)

                    if has_found_tag:
                        # 获取到当前的参数
                        cur_tag_incr = tag_incr_
                    else:
                        # 只是添加到list中
                        cur_tag_incr = dict()
                        tags_inc_list.append(cur_tag_incr)

                    # 这个数据只会使用一次
                    cur_tag_incr.setdefault("incr", 0)
                    cur_tag_incr.setdefault("rela_codes", list())
                    cur_tag_incr.setdefault("tag", tag_)

                    cur_tag_incr['incr'] += 1
                    cur_tag_incr['rela_codes'].append(
                        {"code": code_,
                         "contribution": 1,
                         "name": name_,
                         "up": 10.0,
                         "weight": (round(c_t_status_['cur_up_times'] / (
                             c_t_status_['cur_count'] if c_t_status_['cur_count'] != 0 else -1),
                                          5))})
                    pass
                    # 暂时使用一代替， c_t_status_['cur_count']

                # 选出排名靠前的标签和前几个股票,这部分用于存放到json中
                # if "tags_inc" in temp_calendar_dic:
                #     one_day_code_tag_date_2_db = list()
                #     one_date_calendar_ = temp_calendar_dic['tags_inc']
                #     one_date_calendar_ = sorted(one_date_calendar_, key=lambda x: -x['incr'])
                #     temp_calendar_dic['tags_inc'] = one_date_calendar_ = one_date_calendar_[:5]
                #     for t_ in one_date_calendar_:
                #         one_date_one_tag_calendar_ = t_['rela_codes']
                #         one_date_one_tag_calendar_ = sorted(one_date_one_tag_calendar_, key=lambda x: -x['weight'])
                #         t_['rela_codes'] = one_date_one_tag_calendar_ = one_date_one_tag_calendar_[:9]
                #     code_tag_calendar_list_json.append(temp_calendar_dic)  # {date, tag, rela_codes :[{}]}

                # 记录部分股票的权重信息
                # code_weight_temp_2_db = list()
                # if code_tag_statistic_dic:
                #     for code_ in code_tag_statistic_dic:
                #         for tag_ in code_tag_statistic_dic[code_]:
                #             # up_info
                #             up_info = code_tag_statistic_dic[code_][tag_]
                #             code_weight_temp_2_db.append({'code': code_, 'tag': tag_, 'cur_count': up_info['cur_count'],
                #                                           'cur_up_times': up_info['cur_up_times'],
                #                                           'cur_date': up_info['cur_date'],
                #                                           'cal_date': up_info['cal_date']})
                #             pass

                # 只保存当日涨停的标签的数据到数据库中去保存到db中去
                streamline_statistic_dic = dict()
                for tag_ in temp_tag_num_dic:
                    if temp_tag_num_dic[tag_] >= DAILY_MIN_CODE_NUM:
                        for code_ in code_tag_statistic_dic:
                            for tag_ in code_tag_statistic_dic[code_]:
                                up_info_ = code_tag_statistic_dic[code_][tag_]
                                # 跳过不合格的股票 也就是没有被当当天计算过的
                                if 'cal_date' not in up_info_ or not up_info_['cal_date'] == cur_parse_date_i:
                                    continue
                                streamline_statistic_dic.setdefault(code_, dict())
                                streamline_statistic_dic[code_].setdefault(tag_, dict())
                                streamline_statistic_dic[code_][tag_] = code_tag_statistic_dic[code_][tag_]
                p_tt_nd = dict()
                for tag_ in temp_tag_num_dic:
                    if temp_tag_num_dic[tag_] >= DAILY_MIN_CODE_NUM:
                        p_tt_nd[tag_] = temp_tag_num_dic[tag_]
                print(" tag s:{}".format(sorted(p_tt_nd.items(), key=
                lambda kv: -kv[1])))
                if not int(cur_parse_date_) == end_date:
                    save_data_2_code_tag_weight(streamline_statistic_dic)
                else:
                    save_data_2_code_tag_weight(code_tag_statistic_dic)
        # 保存code_tag_temp_ 信息到数据库中
        # 保存到json
        # save_data_2_file(code_tag_calendar_list_json)
    finally:
        # connection.close()
        pass


if __name__ == '__main__':
    # unittest.main()
    args = sys.argv
    allowed_cmd = ['full_cal', "bulking_cal"]
    cmd = ''
    if len(args) == 1:
        cmd = allowed_cmd[0]
    elif len(args) >= 2:
        cmd = args[1]
    if cmd not in allowed_cmd:
        print("not parse args : {}".format(args))
        exit(0)

    if cmd == 'full_cal':
        main()
    elif cmd == 'bulking_cal':
        main(default_start_date=DATA_START_DATE)
